# 2025-05-02 21:58
# 用deepseek模拟人类对模型的输出进行评估

import itertools
from summarization_framework import load_rsm, Material
from summarization_interface import LinkType, get_idf_value
from syllabus_prerequisite import construct_node_prerequisite_matrix
from pathlib import Path
from collections import Counter, namedtuple
from nltk import word_tokenize
import numpy as np
import re

data_set_dir = Path(__file__).parent.parent.parent / 'datasets'

rsm_text = data_set_dir / 'rsm/sentences'

# deepseek prompt
system_prompt = '''Task Description: Evaluate the syllabus generated by AI for teaching "Resource Space Model (RSM)" and "Foundation of Database" textbooks across four learner levels: Primary, Medium, Advanced, and Expert.

Primary: Fresh university students with no CS background
Medium: CS undergraduates with basic programming/math knowledge
Advanced: Students who completed "Foundation of Database"
Expert: CS graduates who learned RSM

Evaluate the syllabus in the following five dimensions with the following score point:
1. Understandability
- Instructional language/cases match learners' cognitive levels
- Technical terms explained with level-appropriate methods (e.g., basic analogies for Primary, professional terminology allowed for Experts)
- Mathematical formulas/code snippets include difficulty-aligned annotations
- Knowledge density aligns with learners' information processing capacity
- Course objectives logically connect to learners' existing knowledge

2. Coherence
- Clear prerequisite relationships between chapters
- New concepts introduced only after required prior knowledge
- Practical/theoretical content sequencing logic
- Cross-module knowledge bridges for complex topics
- Progressive layering of advanced concepts

3. Coverage
- Includes core domain concepts (benchmarked against authoritative standards)
- Completeness of key theories/technologies/toolchains
- Balanced mandatory/elective content ratio
- Coverage of both foundational skills and higher-order thinking
- Appropriate inclusion of industry前沿 trends

4. Clarity
- Syllabus structure follows cognitive logic (tree/network structures)
- Time allocation reflects topic significance
- Learning objectives use observable action verbs
- Assessment criteria directly map to taught content
- Avoids vague descriptors (e.g., "understand", "be familiar with")

5. Conciseness
- Eliminates redundant instructional content
- Examples demonstrate典型性和代表性
- Removes non-core expansions
- Explanation depth matches learner levels (step-by-step for Primary, conclusion-focused for Experts)
- Optimal knowledge unit granularity (fine-grained for Primary, macro-modules for Experts)

The content of Resource Space Model textbook is as follows:{rsm_text}

The Output Format should follow the json format: 
'''
output_format = '''{
    "Primary": {
        "understandability_score": X,
        "coherence_score": "X",
        "coverage_score": X,
        "clarity_score": "X"
        "conciseness_score": "X"
    },
    "Medium": {
        "understandability_score": X,
        "coherence_score": "X",
        "coverage_score": X,
        "clarity_score": "X"
        "conciseness_score": "X"
    },
    ...
}
'''

user_prompt = 'The generated syllabus is as follows:{syllabus_text}'

from openai import OpenAI
key = 'sk-f7427eb807bd41e49027d5c243a5d00e'
client = OpenAI(api_key=key, base_url="https://api.deepseek.com")

def request_deepseek(syllabus, model_name):
    response = client.chat.completions.create(
        model=model_name,
        messages=[
            {"role": "system", "content": system_prompt},
            {"role": "user", "content": user_prompt.format(syllabus_text=syllabus)},
        ],
        stream=False
    )
    
    # prompt token: 446

    return {
        'relation': response.choices[0].message.content,
        'token': response.usage.completion_tokens
    }

# model_name = 'deepseek-chat'
# model_name = 'deepseek-reasoner'

for model in [
    'bart',
    'luhn',
    'ctrlsumm',
    'lexrank',
    'llama_3d1_8b',
    'llama_3d2_1b',
    'llama_3d2_3b',
    'lsa',
    'qtsumm',
    'textrank',
]:
    syllabus = (data_set_dir / f'rsm/output/{model}/rsm').read_text(encoding='utf-8')
    output_file = Path(__file__).parent / f'figs/deepseek_eval/prompts/{model}.json'
    syllabus = ' '.join(word_tokenize(syllabus)[:1250])

    # output = request_deepseek(syllabus, 'deepseek-chat')
    output_file.write_text(system_prompt.format(rsm_text=syllabus) + user_prompt.format(syllabus_text=syllabus) + output_format, encoding='utf-8')

syllabus = (data_set_dir / f'rsm/wsln_output/focused_node_40_250425_125935/rsm').read_text(encoding='utf-8')
# syllabus = ' '.join(word_tokenize(1250))

# output = request_deepseek(syllabus, 'deepseek-chat')
(Path(__file__).parent / f'figs/deepseek_eval/prompts/wsln.json').write_text(system_prompt.format(rsm_text=syllabus) + user_prompt.format(syllabus_text=syllabus) + output_format, encoding='utf-8')

